{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "866ac389",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "# import igraph as ig\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import time\n",
    "import datetime,time\n",
    "import os\n",
    "import pickle\n",
    "from zhx_config import zhx_config\n",
    "pd.set_option('display.max_info_columns', 500)\n",
    "pd.set_option('display.max_columns', 1000)\n",
    "pd.set_option('display.max_row', 300)\n",
    "pd.set_option('display.float_format', lambda x: ' %.5f' % x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "1f272771",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'初始数据dir': './data/1_初始数据',\n",
       " '预处理后dir': './data/2_预处理后',\n",
       " '特征_dir': './data/3_特征'}"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zhx_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "0ad02f9b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['初始数据.csv']"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 展开数据集列表\n",
    "os.listdir(zhx_config['初始数据dir'] + '/train')\n",
    "os.listdir(zhx_config['初始数据dir'] + '/test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "0369cb21",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_all_csv(path):\n",
    "    return read_all_csv_(path)\n",
    "    \n",
    "def read_all_csv_(path):\n",
    "    data_dict = {}\n",
    "    # print(\"根目录\"+os.getcwd())\n",
    "    for root, ds, fs in os.walk(path):\n",
    "        # print(root)\n",
    "        for f in fs:\n",
    "            if f.endswith('.csv'):\n",
    "                # print(f)\n",
    "                fullname = os.path.join(root, f)\n",
    "                # 去除文件名后缀\n",
    "                data_name = os.path.splitext(f)[0]\n",
    "                data_dict[data_name] = pd.read_csv(fullname)\n",
    "                print(\"读取: \"+fullname)\n",
    "                print(data_dict[data_name].shape)\n",
    "                # \n",
    "    if len(data_dict) == 0:\n",
    "        raise ValueError(\"路径有误, csv文件不存在\")\n",
    "    else:\n",
    "        print(\"文件列表如下:\")\n",
    "        print(data_dict.keys())\n",
    "    return data_dict   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "1e5fedf4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "读取: ./data/1_初始数据/train\\初始数据.csv\n",
      "(9557, 143)\n",
      "文件列表如下:\n",
      "dict_keys(['初始数据'])\n",
      "------------------------------------------------\n",
      "读取: ./data/1_初始数据/test\\初始数据.csv\n",
      "(23856, 142)\n",
      "文件列表如下:\n",
      "dict_keys(['初始数据'])\n"
     ]
    }
   ],
   "source": [
    "# 读取数据集\n",
    "df_train = read_all_csv(zhx_config['初始数据dir'] + '/train')\n",
    "print(\"------------------------------------------------\")\n",
    "df_test = read_all_csv(zhx_config['初始数据dir'] + '/test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "ede295eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['初始数据'].index = df_train['初始数据']['Id'].values\n",
    "df_test['初始数据'].index = df_test['初始数据']['Id'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "7c8a6683",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据清洗\n",
    "def data_cleaning(data):\n",
    "    data['dependency']=np.sqrt(data['SQBdependency'])\n",
    "    data['rez_esc']=data['rez_esc'].fillna(0)\n",
    "    data['v18q1']=data['v18q1'].fillna(0)\n",
    "    data['v2a1']=data['v2a1'].fillna(0)\n",
    "    \n",
    "    conditions = [\n",
    "    (data['edjefe']=='no') & (data['edjefa']=='no'), \n",
    "    (data['edjefe']=='yes') & (data['edjefa']=='no'), \n",
    "    (data['edjefe']=='no') & (data['edjefa']=='yes'), \n",
    "    (data['edjefe']!='no') & (data['edjefe']!='yes') & (data['edjefa']=='no'), \n",
    "    (data['edjefe']=='no') & (data['edjefa']!='no') \n",
    "    ]\n",
    "    choices = [0, 1, 1, data['edjefe'], data['edjefa']]\n",
    "    data['edjefx']=np.select(conditions, choices)\n",
    "    data['edjefx']=data['edjefx'].astype(int)\n",
    "    data.drop(['edjefe', 'edjefa'], axis=1, inplace=True)\n",
    "    \n",
    "    meaneduc_nan=data[data['meaneduc'].isnull()][['Id','idhogar','escolari']]\n",
    "    me=meaneduc_nan.groupby('idhogar')['escolari'].mean().reset_index()\n",
    "    for row in meaneduc_nan.iterrows():\n",
    "        idx=row[0]\n",
    "        idhogar=row[1]['idhogar']\n",
    "        m=me[me['idhogar']==idhogar]['escolari'].tolist()[0]\n",
    "        data.at[idx, 'meaneduc']=m\n",
    "        data.at[idx, 'SQBmeaned']=m*m\n",
    "        \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "9ebc188a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['初始数据'] = data_cleaning(df_train['初始数据'])\n",
    "df_test['初始数据'] = data_cleaning(df_test['初始数据'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "5de6faf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['初始数据'] = df_train['初始数据'].query('parentesco1==1')\n",
    "df_train['初始数据'] = df_train['初始数据'].drop('parentesco1', axis=1)\n",
    "df_test['初始数据'] = df_test['初始数据'].drop('parentesco1', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "b3457919",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_numeric(data, status_name):\n",
    "    status_cols = [s for s in data.columns.tolist() if status_name in s]\n",
    "    print('status column names')\n",
    "    print(status_cols)\n",
    "    status_df = data[status_cols]\n",
    "    status_df.columns = list(range(status_df.shape[1]))\n",
    "    status_numeric = status_df.idxmax(1)\n",
    "    status_numeric.name = status_name\n",
    "    data = pd.concat([data, status_numeric], axis=1)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "7e4d9811",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "status column names\n",
      "['epared1', 'epared2', 'epared3']\n",
      "status column names\n",
      "['epared1', 'epared2', 'epared3']\n",
      "status column names\n",
      "['etecho1', 'etecho2', 'etecho3']\n",
      "status column names\n",
      "['etecho1', 'etecho2', 'etecho3']\n",
      "status column names\n",
      "['eviv1', 'eviv2', 'eviv3']\n",
      "status column names\n",
      "['eviv1', 'eviv2', 'eviv3']\n",
      "status column names\n",
      "['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']\n",
      "status column names\n",
      "['instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']\n"
     ]
    }
   ],
   "source": [
    "status_name_list = ['epared', 'etecho', 'eviv', 'instlevel']\n",
    "for status_name in status_name_list:\n",
    "    for df_now in [df_train['初始数据'] , df_test['初始数据']]:\n",
    "        df_now = get_numeric(df_now, status_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "55ac7372",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>r4t3</th>\n",
       "      <th>tamhog</th>\n",
       "      <th>hogar_total</th>\n",
       "      <th>hhsize</th>\n",
       "      <th>v18q</th>\n",
       "      <th>sanitario1</th>\n",
       "      <th>agesq</th>\n",
       "      <th>mobilephone</th>\n",
       "      <th>area1</th>\n",
       "      <th>female</th>\n",
       "      <th>epared1</th>\n",
       "      <th>epared2</th>\n",
       "      <th>epared3</th>\n",
       "      <th>etecho1</th>\n",
       "      <th>etecho2</th>\n",
       "      <th>etecho3</th>\n",
       "      <th>eviv1</th>\n",
       "      <th>eviv2</th>\n",
       "      <th>eviv3</th>\n",
       "      <th>instlevel1</th>\n",
       "      <th>instlevel2</th>\n",
       "      <th>instlevel3</th>\n",
       "      <th>instlevel4</th>\n",
       "      <th>instlevel5</th>\n",
       "      <th>instlevel6</th>\n",
       "      <th>instlevel7</th>\n",
       "      <th>instlevel8</th>\n",
       "      <th>instlevel9</th>\n",
       "      <th>abastaguafuera</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ID_279628684</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1849</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_f29eb3ddd</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4489</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_68de51c94</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8464</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_ec05b1a7b</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1444</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_1284f8aad</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>900</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_18b0a845b</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>676</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_a31274054</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1600</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_32a00a8bf</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2025</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_79d39dddc</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4489</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID_d45ae367d</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2116</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2973 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              r4t3  tamhog  hogar_total  hhsize  v18q  sanitario1  agesq  \\\n",
       "ID_279628684     1       1            1       1     0           0   1849   \n",
       "ID_f29eb3ddd     1       1            1       1     1           0   4489   \n",
       "ID_68de51c94     1       1            1       1     0           0   8464   \n",
       "ID_ec05b1a7b     4       4            4       4     1           0   1444   \n",
       "ID_1284f8aad     4       4            4       4     0           0    900   \n",
       "...            ...     ...          ...     ...   ...         ...    ...   \n",
       "ID_18b0a845b     5       5            5       5     0           0    676   \n",
       "ID_a31274054     5       5            5       5     0           0   1600   \n",
       "ID_32a00a8bf     5       5            5       5     0           0   2025   \n",
       "ID_79d39dddc     2       2            2       2     0           0   4489   \n",
       "ID_d45ae367d     5       5            5       5     0           0   2116   \n",
       "\n",
       "              mobilephone  area1  female  epared1  epared2  epared3  etecho1  \\\n",
       "ID_279628684            1      1       0        0        1        0        1   \n",
       "ID_f29eb3ddd            1      1       0        0        1        0        0   \n",
       "ID_68de51c94            0      1       1        0        1        0        0   \n",
       "ID_ec05b1a7b            1      1       0        0        0        1        0   \n",
       "ID_1284f8aad            1      1       0        1        0        0        1   \n",
       "...                   ...    ...     ...      ...      ...      ...      ...   \n",
       "ID_18b0a845b            1      0       1        0        1        0        0   \n",
       "ID_a31274054            1      0       0        0        1        0        1   \n",
       "ID_32a00a8bf            1      0       0        0        1        0        0   \n",
       "ID_79d39dddc            1      0       0        0        0        1        0   \n",
       "ID_d45ae367d            1      0       0        0        1        0        0   \n",
       "\n",
       "              etecho2  etecho3  eviv1  eviv2  eviv3  instlevel1  instlevel2  \\\n",
       "ID_279628684        0        0      1      0      0           0           0   \n",
       "ID_f29eb3ddd        1        0      0      1      0           0           0   \n",
       "ID_68de51c94        0        1      0      0      1           0           0   \n",
       "ID_ec05b1a7b        0        1      0      0      1           0           0   \n",
       "ID_1284f8aad        0        0      0      1      0           0           0   \n",
       "...               ...      ...    ...    ...    ...         ...         ...   \n",
       "ID_18b0a845b        1        0      0      1      0           0           1   \n",
       "ID_a31274054        0        0      0      1      0           0           1   \n",
       "ID_32a00a8bf        1        0      0      1      0           0           1   \n",
       "ID_79d39dddc        0        1      0      0      1           1           0   \n",
       "ID_d45ae367d        1        0      0      1      0           0           0   \n",
       "\n",
       "              instlevel3  instlevel4  instlevel5  instlevel6  instlevel7  \\\n",
       "ID_279628684           0           1           0           0           0   \n",
       "ID_f29eb3ddd           0           0           0           0           0   \n",
       "ID_68de51c94           0           0           1           0           0   \n",
       "ID_ec05b1a7b           0           0           1           0           0   \n",
       "ID_1284f8aad           0           1           0           0           0   \n",
       "...                  ...         ...         ...         ...         ...   \n",
       "ID_18b0a845b           0           0           0           0           0   \n",
       "ID_a31274054           0           0           0           0           0   \n",
       "ID_32a00a8bf           0           0           0           0           0   \n",
       "ID_79d39dddc           0           0           0           0           0   \n",
       "ID_d45ae367d           0           1           0           0           0   \n",
       "\n",
       "              instlevel8  instlevel9  abastaguafuera  \n",
       "ID_279628684           0           0               0  \n",
       "ID_f29eb3ddd           1           0               0  \n",
       "ID_68de51c94           0           0               0  \n",
       "ID_ec05b1a7b           0           0               0  \n",
       "ID_1284f8aad           0           0               0  \n",
       "...                  ...         ...             ...  \n",
       "ID_18b0a845b           0           0               1  \n",
       "ID_a31274054           0           0               0  \n",
       "ID_32a00a8bf           0           0               0  \n",
       "ID_79d39dddc           0           0               0  \n",
       "ID_d45ae367d           0           0               0  \n",
       "\n",
       "[2973 rows x 29 columns]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train['初始数据'][['r4t3', 'tamhog', 'hogar_total', 'hhsize', 'v18q', 'sanitario1', 'agesq',\n",
    "                    'mobilephone', 'area1', 'female', 'epared1', 'epared2',\n",
    "                    'epared3', 'etecho1', 'etecho2', 'etecho3',\n",
    "                    'eviv1', 'eviv2', 'eviv3', 'instlevel1', 'instlevel2',\n",
    "                    'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6',\n",
    "                    'instlevel7', 'instlevel8', 'instlevel9', 'abastaguafuera']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "6c4e2fd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "needless_cols = ['r4t3', 'tamhog', 'hogar_total', 'hhsize', 'v18q', 'sanitario1', 'agesq',\n",
    "                'mobilephone', 'area1', 'female', 'epared1', 'epared2',\n",
    "                'epared3', 'etecho1', 'etecho2', 'etecho3',\n",
    "                'eviv1', 'eviv2', 'eviv3', 'instlevel1', 'instlevel2',\n",
    "                'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6',\n",
    "                'instlevel7', 'instlevel8', 'instlevel9', 'abastaguafuera']\n",
    "SQB_cols = [s for s in df_train['初始数据'].columns.tolist() if 'SQB' in s]\n",
    "parentesco_cols = [s for s in df_train['初始数据'].columns.tolist() if 'parentesco' in s]\n",
    "\n",
    "needless_cols.extend(SQB_cols)\n",
    "needless_cols.extend(parentesco_cols)\n",
    "\n",
    "df_train['初始数据'] = df_train['初始数据'].drop(needless_cols, axis=1)\n",
    "df_test['初始数据'] = df_test['初始数据'].drop(needless_cols, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "3f128e89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "feature columns \n",
      " 140 -> 90\n"
     ]
    }
   ],
   "source": [
    "ori_train = pd.read_csv(os.path.join(zhx_config['初始数据dir'] + '/train/' + '/初始数据.csv'))\n",
    "ori_train_X = ori_train.drop(['Id', 'Target', 'idhogar'], axis=1)\n",
    "\n",
    "train_X = df_train['初始数据'].drop(['Id', 'Target', 'idhogar'], axis=1)\n",
    "\n",
    "print('feature columns \\n {} -> {}'.format(ori_train_X.shape[1], train_X.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "8b81d6ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_Id = df_train['初始数据']['Id']\n",
    "train_idhogar = df_train['初始数据']['idhogar']\n",
    "train_y = df_train['初始数据']['Target']\n",
    "train_X = df_train['初始数据'].drop(['Id', 'Target', 'idhogar'], axis=1)\n",
    "\n",
    "test_Id = df_test['初始数据']['Id']\n",
    "test_idhogar = df_test['初始数据']['idhogar']\n",
    "test_X = df_test['初始数据'].drop(['Id', 'idhogar'], axis=1)\n",
    "\n",
    "all_Id = pd.concat([train_Id, test_Id], axis=0, sort=False)\n",
    "all_idhogar = pd.concat([train_idhogar, test_idhogar], axis=0, sort=False)\n",
    "all_X = pd.concat([train_X, test_X], axis=0, sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "f85dbeaf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier()"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.metrics import confusion_matrix, f1_score, make_scorer\n",
    "import lightgbm as lgb\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(train_X, train_y, test_size=0.1, random_state=0)\n",
    "\n",
    "F1_scorer = make_scorer(f1_score, greater_is_better=True, average='macro')\n",
    "\n",
    "gbm = lgb.LGBMClassifier()\n",
    "\n",
    "gbm.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "db134602",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = gbm.predict(test_X)\n",
    "pred = pd.Series(data=pred, index=test_Id.values, name='Target')\n",
    "pred = pd.concat([test_Id, pred], axis=1)\n",
    "submission = pred\n",
    "submission.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "be265cc0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix: \n",
      " [[  5   7   1   9]\n",
      " [  3  14   4  27]\n",
      " [  3   7   3  25]\n",
      " [  1   7   7 175]]\n",
      "macro F1 score: \n",
      " 0.39156770898761295\n"
     ]
    }
   ],
   "source": [
    "y_test_pred = gbm.predict(X_test)\n",
    "cm = confusion_matrix(y_test, y_test_pred)\n",
    "f1 = f1_score(y_test, y_test_pred, average='macro')\n",
    "print(\"confusion matrix: \\n\", cm)\n",
    "print(\"macro F1 score: \\n\", f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca239d13",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "aae6525c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d29e96f8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "e2607d3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "table_name = '初始数据'\n",
    "dir = zhx_config['预处理后dir']\n",
    "\n",
    "pickle.dump(df_train[table_name], open(dir + '/train/' + table_name + '.pkl', 'wb'))\n",
    "pickle.dump(df_test[table_name], open(dir + '/test/' + table_name + '.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e88836e8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
